print "hello"
print "Begins..."

documentOrientedTupleSetList = []

inputFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/100KQueries_tail_100_top_10000_results"
inputFileHandler = open(inputFileName,"r")

currentLine = inputFileHandler.readline()
nextLine = ""
currentQID = ""

numOfResultsForTheAllQueries = 0
numOfQueriesHavingQID = 0
numOfQueriesHavingSearchContent = 0
numOfQueriesHavingSearchResults = 0

while currentLine:
    # sampling parsing line:
    # qid: 701
    if currentLine.startswith("qid:"):
        currentQID = currentLine.strip().split(":")[1].strip()
        print "currentQID:",currentQID
        numOfQueriesHavingQID += 1
    
    # sample parsing line:
    # Search: u s  oil industry history
    if currentLine.startswith("Search:"):
        elementList = currentLine.strip().split(" ")
        
        currentSearchContent = ""
        
        queryTermList = []
        for element in elementList[1:]:
            if element.strip() != "":
                queryTermList.append( element.strip() )        
        currentQueryLength = len(queryTermList)
        print "currentQueryLength:",currentQueryLength

        for element in queryTermList:
            currentSearchContent += element + " "
        
        print currentSearchContent
        numOfQueriesHavingSearchContent += 1
        
        # the sampling line will be like the following format:
        # [2013-03-12 13:49:35] 38871 terms with their lists length have been loaded.
        # So I just pass and ignore them
        thisLineHasNoUse = inputFileHandler.readline()
        
        currentQueryTermIndexDict = {}
        nextLine = inputFileHandler.readline()
        #print nextLine.strip()
        if nextLine.strip() != "":
            nextLineElements = nextLine.strip().split(" ")
            # print "nextLineElements:",nextLineElements
            # sample parsing line: 
            # oil:0 industry:1 history:2 u:3 s:4
            for element in nextLineElements:
                if element.split(":")[0] not in currentQueryTermIndexDict:
                    currentQueryTermIndexDict[ int(element.split(":")[1]) ] = element.split(":")[0]
            
            print "currentQueryTermIndexDict:",currentQueryTermIndexDict
            # print "len(currentQueryTermIndexDict):",len(currentQueryTermIndexDict)
            
            
            
            # Now, it is time to read the threshold line
            currentLine = inputFileHandler.readline()
            if len( currentLine.strip().split(":") ) == 2:
                print currentLine.strip().split(":")[0],currentLine.strip().split(":")[1]
            else:
                print "format problem"
                exit()
            
            # Now, it is time to read the computation method line
            currentLine = inputFileHandler.readline()
            if len( currentLine.strip().split(":") ) == 2:
                print currentLine.strip().split(":")[0],currentLine.strip().split(":")[1]
            else:
                print "format problem"
                exit()
                        
            # Now, it is time to construct the training examples
            currentLine = inputFileHandler.readline()
            numOfResultsForTheCurrentQuery = 0
            currentResultLines = []
            # print "currentLine:",currentLine
            print "currentLine:",currentLine
            lineElements = currentLine.strip().split(" ")
            while len(lineElements) == 4:
                numOfResultsForTheCurrentQuery += 1
                currentResultLines.append(currentLine.strip())
                
                
                rank_in_the_result_list = int( lineElements[0] )
                score_in_the_result_list = float( lineElements[1] )
                docID_in_the_result_list = lineElements[2]
                trecID_in_the_result_list = lineElements[3]
                
                tuple = (currentQID,rank_in_the_result_list,score_in_the_result_list,docID_in_the_result_list,trecID_in_the_result_list)
                documentOrientedTupleSetList.append(tuple)
                
                currentLine = inputFileHandler.readline()
                lineElements = currentLine.strip().split(" ")
               
            numOfResultsForTheAllQueries += numOfResultsForTheCurrentQuery
            
            # sampling method 1: among all the top10 documents, select those documents, open those documents and extract the corresponding postings.
            # This method make sure that once the document have been selected, ALL the coresponding postings will also be selected.
            # This sampling method has been developed on 2013/01/09
            
            print "numOfResultsForTheCurrentQuery:",numOfResultsForTheCurrentQuery
            print "len(currentResultLines):",len(currentResultLines)
            # print "len(queryTermList):",len(queryTermList)
            
            if len(queryTermList) > 10:
                # The result list returned only have 10 holds so it only support the query which has the maximun length of 10
                pass
            else:
                if numOfResultsForTheCurrentQuery == 0:
                    pass
                    # print "currentQID:",currentQID
                    # print "currentSearchContent:",currentSearchContent
                    # print "numOfQueriesHavingSearchContent:",numOfQueriesHavingSearchContent
                    # print "numOfResultsForTheCurrentQuery:",numOfResultsForTheCurrentQuery
                else:
                    numOfQueriesHavingSearchResults += 1
            print
    
    currentLine = inputFileHandler.readline()


print "numOfQueriesHavingQID:",numOfQueriesHavingQID
print "numOfQueriesHavingSearchContent:",numOfQueriesHavingSearchContent
print "numOfQueriesHavingSearchResults:",numOfQueriesHavingSearchResults
inputFileHandler.close()


outputFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/small_set_of_gov2_documents_needed_to_be_extracted_sorted_by_qid"
outputFileHandler = open(outputFileName,"w")

for tuple in documentOrientedTupleSetList:
    (currentQID,rank_in_the_result_list,score_in_the_result_list,docID_in_the_result_list,trecID_in_the_result_list) = tuple
    outputFileHandler.write(currentQID + " ")
    outputFileHandler.write(str(rank_in_the_result_list) + " ")
    outputFileHandler.write(str(score_in_the_result_list) + " ")
    outputFileHandler.write(str(docID_in_the_result_list) + " ")
    outputFileHandler.write(str(trecID_in_the_result_list) + " ")
    outputFileHandler.write("\n")

outputFileHandler.close()

print "Ends."





















